import re,sys,os,socket
import gzip,os,string,tempfile
from urllib2 import *
from threading import Thread
from Queue import Queue
import time,gzip,tempfile,string,random


class handler:
  def __init__(self):
    self.ext="jpg"
    self.opener=build_opener()
  def decompress_if_gzip(self,page):
    if page[0] not in string.printable:
      # in gzip formart
      fname = tempfile.mktemp()
      f=file(fname,"wb")
      f.write(page)
      f.close()
      gf = gzip.open(fname,"rb")
      page = gf.read()
      gf.close()
      os.unlink(fname)
    return page

  def write_log(self,s):
    log_file=file("log.txt","ab")
    s=time.strftime("[%d/%m/%y %H:%M:%S]\t")+s
    print s
    try:
      log_file.write(s.encode('utf-8')+"\n")
    except Exception as e:
      log_file.write("we encounter an encoding problem..\n")
    log_file.close()
    return s
  def try_to_read_url(self,url, times_to_try=5, header={}):
    while times_to_try > 0:
      try:
        r=Request(url)
        for a,b in header.items():
          r.add_header(a,b)
        res=self.opener.open(r).read()
        break
      except IOError:
        times_to_try-=1
        self.write_log("time out when downloading %s, %d times left to try" % (url,times_to_try))
      except Exception as e:
        print e
        self.write_log("unknown error, download terminates")
        times_to_try=0
      time.sleep(2)

    if times_to_try==0:
      self.write_log("fail to download %s" % url)
      return None
    else:
      return res

  def download_img(self,url,file_name):
    img=self.try_to_read_url(url,5)
    if img==None:return []
    f=file(file_name,"wb")
    f.write(img)
    f.close()
    self.write_log( "%s ok" % file_name )
  def get_dir_name(self,dir,url):
    if not dir.endswith(os.path.sep):
      dir+=os.path.sep
    if url.endswith("/"):url=url[:-1]
    dir_name=re.findall(r"/([^/]*)$",url)[0]
    if dir_name.endswith(".html"): dir_name=dir_name[:-5]
    dir_name=re.sub("[^0-9a-zA-Z-_]","",dir_name)

    return dir+dir_name+os.sep
